Importing libraries and required files¶
In [1]:
import pandas as pd
import numpy as np
import plotly.graph_objs as go
import plotly.offline as offline
import itertools
from sklearn.metrics import mean_squared_error
from math import sqrt
import matplotlib.pyplot as plt
import seaborn as sns
import datetime
import statsmodels.api as sm
import matplotlib
import warnings
warnings.filterwarnings("ignore")
from statsmodels.tsa.stattools import adfuller
offline.init_notebook_mode()
xls=pd.ExcelFile("PET_PRI_SPT_S1_M.xls")
df_crudeOil=pd.read_excel(xls,"Data 1")
df_cGasoline=pd.read_excel(xls,"Data 2")
df_rGasoline=pd.read_excel(xls,"Data 3")
df_heatOil=pd.read_excel(xls,"Data 4")
df_diesel=pd.read_excel(xls,"Data 5")
df_jet=pd.read_excel(xls,"Data 6")
df_prop=pd.read_excel(xls,"Data 7")
Cleaning Datasets¶
In [2]:
def cleanData(data):
data.columns=data.iloc[1,:]
data=data.iloc[2:,:]
return data
In [3]:
#Crude Oil Dataset
df_crudeOil=cleanData(df_crudeOil)
df_crudeOil.head()
Out[3]:
In [4]:
#C Gasoline Dataset
df_cGasoline=cleanData(df_cGasoline)
df_cGasoline.head()
Out[4]:
In [5]:
#R Gasoline Dataset
df_rGasoline=cleanData(df_rGasoline)
df_rGasoline.head()
Out[5]:
In [6]:
#Heat Oil Dataset
df_heatOil=cleanData(df_heatOil)
df_heatOil.head()
Out[6]:
In [7]:
#Diesel Dataset
df_diesel=cleanData(df_diesel)
df_diesel.head()
Out[7]:
In [8]:
#Jet Fuel Dataset
df_jet=cleanData(df_jet)
df_jet.head()
Out[8]:
In [9]:
#Propane Dataset
df_prop=cleanData(df_prop)
df_prop.head()
Out[9]:
Creating column in each dataset givem to find price changes month over month¶
In [10]:
#Crude Oil
df_crudeOil['Cushing, OK WTI Monthly Change']=df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)'].pct_change()*100
df_crudeOil['Europe Brent Monthly Change']=df_crudeOil['Europe Brent Spot Price FOB (Dollars per Barrel)'].pct_change()*100
df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)']=df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)'].astype('float')
df_crudeOil['Europe Brent Spot Price FOB (Dollars per Barrel)']=df_crudeOil['Europe Brent Spot Price FOB (Dollars per Barrel)'].astype('float')
df_crudeOil.Date = pd.to_datetime(df_crudeOil.Date)
df_crudeOil.set_index('Date', inplace=True)
df_crudeOil
Out[10]:
In [11]:
#Converntional Gasoline
df_cGasoline['New York Harbor Conventional Gasoline Monthly Change']=df_cGasoline['New York Harbor Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)'].pct_change()*100
df_cGasoline['U.S. Gulf Coast Conventional Gasoline Monthly Change']=df_cGasoline['U.S. Gulf Coast Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)'].pct_change()*100
df_cGasoline['New York Harbor Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)']=df_cGasoline['New York Harbor Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)'].astype('float')
df_cGasoline['U.S. Gulf Coast Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)']=df_cGasoline['U.S. Gulf Coast Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)'].astype('float')
df_cGasoline.Date = pd.to_datetime(df_cGasoline.Date)
df_cGasoline.set_index('Date', inplace=True)
df_cGasoline
Out[11]:
In [12]:
#Regular Gasoline
df_rGasoline['Los Angeles Reformulated RBOB Regular Gasoline Monthly Change']=df_rGasoline['Los Angeles Reformulated RBOB Regular Gasoline Spot Price (Dollars per Gallon)'].pct_change()*100
df_rGasoline['Los Angeles Reformulated RBOB Regular Gasoline Spot Price (Dollars per Gallon)']=df_rGasoline['Los Angeles Reformulated RBOB Regular Gasoline Spot Price (Dollars per Gallon)'].astype('float')
df_rGasoline.Date = pd.to_datetime(df_rGasoline.Date)
df_rGasoline.set_index('Date', inplace=True)
df_rGasoline
Out[12]:
In [13]:
#Heat Oil
df_heatOil['New York Harbor No. 2 Heating Oil Monthly Change']=df_heatOil['New York Harbor No. 2 Heating Oil Spot Price FOB (Dollars per Gallon)'].pct_change()*100
df_heatOil['New York Harbor No. 2 Heating Oil Spot Price FOB (Dollars per Gallon)']=df_heatOil['New York Harbor No. 2 Heating Oil Spot Price FOB (Dollars per Gallon)'].astype('float')
df_heatOil.Date = pd.to_datetime(df_heatOil.Date)
df_heatOil.set_index('Date', inplace=True)
df_heatOil
Out[13]:
In [14]:
#Diesel Fuel
df_diesel['New York Harbor Monthly Change']=df_diesel['New York Harbor Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)'].pct_change()*100
df_diesel['U.S. Gulf Coast Monthly Change']=df_diesel['U.S. Gulf Coast Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)'].pct_change()*100
df_diesel['Los Angeles, CA Monthly Change']=df_diesel['Los Angeles, CA Ultra-Low Sulfur CARB Diesel Spot Price (Dollars per Gallon)'].pct_change()*100
df_diesel['New York Harbor Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)']=df_diesel['New York Harbor Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)'].astype('float')
df_diesel['U.S. Gulf Coast Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)']=df_diesel['U.S. Gulf Coast Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)'].astype('float')
df_diesel['Los Angeles, CA Ultra-Low Sulfur CARB Diesel Spot Price (Dollars per Gallon)']=df_diesel['Los Angeles, CA Ultra-Low Sulfur CARB Diesel Spot Price (Dollars per Gallon)'].astype('float')
df_diesel.Date = pd.to_datetime(df_diesel.Date)
df_diesel.set_index('Date', inplace=True)
df_diesel
Out[14]:
In [15]:
#Jet Fuel
df_jet['U.S. Gulf Coast Monthly Change']=df_jet['U.S. Gulf Coast Kerosene-Type Jet Fuel Spot Price FOB (Dollars per Gallon)'].pct_change()*100
df_jet['U.S. Gulf Coast Kerosene-Type Jet Fuel Spot Price FOB (Dollars per Gallon)']=df_jet['U.S. Gulf Coast Kerosene-Type Jet Fuel Spot Price FOB (Dollars per Gallon)'].astype('float')
df_jet.Date = pd.to_datetime(df_jet.Date)
df_jet.set_index('Date', inplace=True)
df_jet
Out[15]:
In [16]:
#Propane
df_prop['Mont Belvieu, TX Propane Monthly Change']=df_prop['Mont Belvieu, TX Propane Spot Price FOB (Dollars per Gallon)'].pct_change()*100
df_prop['Mont Belvieu, TX Propane Spot Price FOB (Dollars per Gallon)']=df_prop['Mont Belvieu, TX Propane Spot Price FOB (Dollars per Gallon)'].astype('float')
df_prop.Date = pd.to_datetime(df_prop.Date)
df_prop.set_index('Date', inplace=True)
df_prop
Out[16]:
Summarized price and price movement by year for all petrolum products¶
In [17]:
def getYearlySummary(data):
return data.resample('Y').mean()
In [18]:
#Crude Oil Yearly Summarized Table
getYearlySummary(df_crudeOil)
Out[18]:
In [19]:
#Conventional Gasoline Yearly Summarized Table
getYearlySummary(df_cGasoline)
Out[19]:
In [20]:
#Regular Gasoline Yearly Summarized Table
getYearlySummary(df_rGasoline)
Out[20]:
In [21]:
#Heat Oil Yearly Summarized Table
getYearlySummary(df_heatOil)
Out[21]:
In [22]:
#Diesel Yearly Summarized Table
getYearlySummary(df_diesel)
Out[22]:
In [23]:
#Jet Fuel Yearly Summarized Table
getYearlySummary(df_jet)
Out[23]:
In [24]:
#Propane Yearly Summarized Table
getYearlySummary(df_prop)
Out[24]:
Exploratory Analysis¶
In [25]:
#Functions needed for Exploratory Analysis
def getImportantMonths(data):
comments=[]
for i in data.columns:
comments.append(str(i))
comments.append("min: "+str(min(data[i].fillna(9999)))+" in "+str(data[data[i] == min(data[i].fillna(9999))].index.tolist()[0]))
comments.append("max: "+str(max(data[i].fillna(-1)))+" in "+str(data[data[i] == max(data[i].fillna(-1))].index.tolist()[0]))
return comments
def getDistribution(data):
for i in data.columns:
print("Distribution of "+str(i))
plt.hist(data[i])
plt.show()
Crude Oil Analysis
In [26]:
#Crude Oil Data Visualizations
trace0 = go.Scatter(
x = df_crudeOil.index,
y = df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)'],
name = 'Cushing, OK WTI',
line = dict(
color = ('green'),
width = 2)
)
trace1 = go.Scatter(
x = df_crudeOil.index,
y = df_crudeOil['Europe Brent Spot Price FOB (Dollars per Barrel)'],
name = 'Europe Brent',
line = dict(
color = ('orange'),
width = 2,)
)
data = [trace0, trace1]
layout = dict(title = 'Crude Oil Analysis',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Barrel'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
In [27]:
df_crudeOil.describe()
Out[27]:
In [28]:
getImportantMonths(df_crudeOil)
Out[28]:
In [29]:
getDistribution(df_crudeOil)
In [30]:
cmap = sns.color_palette("Blues")
sns.heatmap(df_crudeOil.iloc[:,:2],cmap=cmap)
Out[30]:
Conventional Gasoline
In [31]:
trace0 = go.Scatter(
x = df_cGasoline.index,
y = df_cGasoline['New York Harbor Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)'],
name = 'New York Harbor',
line = dict(
color = ('green'),
width = 2)
)
trace1 = go.Scatter(
x = df_cGasoline.index,
y = df_cGasoline['U.S. Gulf Coast Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)'],
name = 'U.S. Gulf Coast',
line = dict(
color = ('orange'),
width = 2,)
)
data = [trace0, trace1]
layout = dict(title = 'Conventional Gasoline Analysis',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Gallon'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
In [32]:
df_cGasoline.describe()
Out[32]:
In [33]:
getImportantMonths(df_cGasoline)
Out[33]:
In [34]:
getDistribution(df_cGasoline)
In [35]:
cmap = sns.color_palette("Blues")
sns.heatmap(df_cGasoline.iloc[:,:2],cmap=cmap)
Out[35]:
Regular Gasoline
In [36]:
trace0 = go.Scatter(
x = df_rGasoline.index,
y = df_rGasoline['Los Angeles Reformulated RBOB Regular Gasoline Spot Price (Dollars per Gallon)'],
name = 'New York Harbor',
line = dict(
color = ('green'),
width = 2)
)
data = [trace0]
layout = dict(title = 'Regular Gasoline Analysis',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Gallon'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
In [37]:
df_rGasoline.describe()
Out[37]:
In [38]:
getImportantMonths(df_rGasoline)
Out[38]:
In [39]:
getDistribution(df_rGasoline)
Heat Oil Analyses
In [40]:
#Heat Oil
trace0 = go.Scatter(
x = df_heatOil.index,
y = df_heatOil['New York Harbor No. 2 Heating Oil Spot Price FOB (Dollars per Gallon)'],
name = 'New York Harbor',
line = dict(
color = ('green'),
width = 2)
)
data = [trace0]
layout = dict(title = 'Heat Oil Analysis',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Gallon'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
In [41]:
df_heatOil.describe()
Out[41]:
In [42]:
getImportantMonths(df_heatOil)
Out[42]:
In [43]:
getDistribution(df_heatOil)
Diesel Oil Price Analysis
In [44]:
#Diesel Oil
trace0 = go.Scatter(
x = df_diesel.index,
y = df_diesel['New York Harbor Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)'],
name = 'New York Harbor',
line = dict(
color = ('rgb(228,26,28)'),
width = 2)
)
trace1 = go.Scatter(
x = df_diesel.index,
y = df_diesel['U.S. Gulf Coast Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)'],
name = 'U.S. Gulf Coast',
line = dict(
color = ('rgb(55,126,184)'),
width = 2,)
)
trace2 = go.Scatter(
x = df_diesel.index,
y = df_diesel['Los Angeles, CA Ultra-Low Sulfur CARB Diesel Spot Price (Dollars per Gallon)'],
name = 'Los Angeles, CA',
line = dict(
color = ('rgb(77,175,74)'),
width = 2,)
)
data = [trace0, trace1,trace2]
layout = dict(title = 'Diesel Oil Analysis',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Gallon'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
In [45]:
df_diesel.describe()
Out[45]:
In [46]:
getImportantMonths(df_diesel)
Out[46]:
In [47]:
getDistribution(df_diesel)
Jet Fuel Analysis
In [48]:
#Jet Fuel
trace0 = go.Scatter(
x = df_jet.index,
y = df_jet['U.S. Gulf Coast Kerosene-Type Jet Fuel Spot Price FOB (Dollars per Gallon)'],
name = 'U.S. Gulf Coast',
line = dict(
color = ('green'),
width = 2)
)
data = [trace0]
layout = dict(title = 'Jet Fuel Analysis',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Gallon'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
In [49]:
df_jet.describe()
Out[49]:
In [50]:
getImportantMonths(df_jet)
Out[50]:
In [51]:
getDistribution(df_jet)
Propane Analysis
In [52]:
#Propane
trace0 = go.Scatter(
x = df_prop.index,
y = df_prop['Mont Belvieu, TX Propane Spot Price FOB (Dollars per Gallon)'],
name = 'Mont Belvieu, TX',
line = dict(
color = ('green'),
width = 2)
)
data = [trace0]
layout = dict(title = 'Propane Analysis',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Gallon'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
In [53]:
df_prop.describe()
Out[53]:
In [54]:
getImportantMonths(df_prop)
Out[54]:
In [55]:
getDistribution(df_prop)
Let's club all the datasets to get a view of the price of all of them combinedly¶
Transforming the unit Barrel to Gallon price of Crude Oil dataset to make proper comparison between the other datasets
In [56]:
df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Gallon)']=df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)'].apply(lambda x: x/31.5)
df_crudeOil['Europe Brent Spot Price FOB (Dollars per Gallon)']=df_crudeOil['Europe Brent Spot Price FOB (Dollars per Barrel)'].apply(lambda x: x/31.5)
df_crudeOil
Out[56]:
Visualizing all time series data together
In [57]:
trace0 = go.Scatter(
x = df_crudeOil.index,
y = df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Gallon)'],
name = 'Cushing, OK WTI-Crude Oil',
line = dict(
color = ('rgb(166,206,227)'),
width = 2)
)
trace1 = go.Scatter(
x = df_crudeOil.index,
y = df_crudeOil['Europe Brent Spot Price FOB (Dollars per Gallon)'],
name = 'Europe Brent-Crude Oil',
line = dict(
color = ('rgb(31,120,180)'),
width = 2,)
)
trace2 = go.Scatter(
x = df_cGasoline.index,
y = df_cGasoline['New York Harbor Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)'],
name = 'New York Harbor-Conventional Gasoline',
line = dict(
color = ('rgb(178,223,138)'),
width = 2)
)
trace3 = go.Scatter(
x = df_cGasoline.index,
y = df_cGasoline['U.S. Gulf Coast Conventional Gasoline Regular Spot Price FOB (Dollars per Gallon)'],
name = 'U.S. Gulf Coast-Conventional Gasoline',
line = dict(
color = ('rgb(51,160,44)'),
width = 2,)
)
trace4 = go.Scatter(
x = df_rGasoline.index,
y = df_rGasoline['Los Angeles Reformulated RBOB Regular Gasoline Spot Price (Dollars per Gallon)'],
name = 'Los Angeles-RBOB Regular Gasoline',
line = dict(
color = ('rgb(255,255,153)'),
width = 2,)
)
trace5 = go.Scatter(
x = df_heatOil.index,
y = df_heatOil['New York Harbor No. 2 Heating Oil Spot Price FOB (Dollars per Gallon)'],
name = 'New York Harbor-Heating Oil',
line = dict(
color = ('rgb(177,89,40)'),
width = 2,)
)
trace6 = go.Scatter(
x = df_diesel.index,
y = df_diesel['New York Harbor Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)'],
name = 'New York Harbor-Diesel',
line = dict(
color = ('rgb(253,191,111)'),
width = 2,)
)
trace7 = go.Scatter(
x = df_diesel.index,
y = df_diesel['U.S. Gulf Coast Ultra-Low Sulfur No 2 Diesel Spot Price (Dollars per Gallon)'],
name = 'U.S. Gulf Coast-Diesel',
line = dict(
color = ('rgb(255,127,0)'),
width = 2,)
)
trace8 = go.Scatter(
x = df_diesel.index,
y = df_diesel['Los Angeles, CA Ultra-Low Sulfur CARB Diesel Spot Price (Dollars per Gallon)'],
name = 'Los Angeles, CA-Diesel',
line = dict(
color = ('rgb(227,26,28)'),
width = 2,)
)
trace9 = go.Scatter(
x = df_jet.index,
y = df_jet['U.S. Gulf Coast Kerosene-Type Jet Fuel Spot Price FOB (Dollars per Gallon)'],
name = 'U.S. Gulf Coast-Jet Fuel',
line = dict(
color = ('rgb(251,154,153)'),
width = 2,)
)
trace10 = go.Scatter(
x = df_prop.index,
y = df_prop['Mont Belvieu, TX Propane Spot Price FOB (Dollars per Gallon)'],
name = 'Mont Belvieu, TX-Propane',
line = dict(
color = ('rgb(106,61,154)'),
width = 2,)
)
data = [trace0, trace1,trace2,trace3,trace4,trace5,trace6,trace7,trace8,trace9,trace10]
layout = dict(title = 'Analysis of All Pretroleum fuels',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Gallon'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
Creating heatmap to see the price distribution of products across the years
In [58]:
combined_data = pd.concat([df_crudeOil.iloc[:,4:], df_cGasoline.iloc[:,:2],df_rGasoline.iloc[:,:1],df_heatOil
.iloc[:,:1],df_diesel.iloc[:,:3],df_jet.iloc[:,:1],df_prop.iloc[:,:1]], axis=1, sort=False)
cmap = sns.color_palette("Blues")
sns.heatmap(combined_data,cmap=cmap)
Out[58]:
Plotting the coorelation plot to see relation between all the variables
In [59]:
combined_data.columns=[['Crushing Crude','Eurupe Crude','NY C Gas','Gulf Coast C Gas','LA R Gas','NY Heat','NY Diesel','Gulf Coast Diesel','LA Diesel','Gulf Coast Kerosene','TX Prop']]
g = sns.pairplot(combined_data)
In [60]:
#Montly Change HeatMap
combined_data_monthly_change = pd.concat([df_crudeOil.iloc[:,2:4], df_cGasoline.iloc[:,2:],df_rGasoline.iloc[:,1:],df_heatOil
.iloc[:,1:],df_diesel.iloc[:,3:],df_jet.iloc[:,1:],df_prop.iloc[:,1:]], axis=1, sort=False)
cmap = sns.color_palette("Blues")
sns.heatmap(combined_data_monthly_change,cmap=cmap)
Out[60]:
Identification the correlation between Conventional Gasoline prices between New York Harbor and U.S. Gulf Coast¶
In [61]:
g = sns.pairplot(df_cGasoline.iloc[:,:2])
In [62]:
np.corrcoef(df_cGasoline.iloc[:,0],df_cGasoline.iloc[:,1])
Out[62]:
We can see strong positive coorelation between both the cities' prices for conventional Gasoline
Finding out a timeframe where prices of all Products were lower than prior data points¶
In [63]:
def checkIfLowerThanPrior(prior,current):
to_check=current-prior
if ((to_check <0).sum() == to_check.size).astype(np.int)==1:
return True
else:
return False
In [64]:
#Checking for monthly data
timeframe=[]
for row in range(0,len(combined_data)):
if np.isnan(np.array(combined_data.iloc[row-1,:])).any() or np.isnan(np.array(combined_data.iloc[row,:])).any():
continue
else:
if checkIfLowerThanPrior(np.array(combined_data.iloc[row-1,:]),np.array(combined_data.iloc[row,:])):
timeframe.append(combined_data.index[row])
#Plot to represent the findings
print("Plot to represent the times when all Products prices were lower than their prior data")
X = pd.Series(timeframe).apply(lambda x: str(x).split('-')[1])
Y = pd.Series(timeframe).apply(lambda x: str(x).split('-')[0])
Z = np.ones(len(X))
plot_data = pd.DataFrame({'Month': X, 'Year': Y, 'True/False': Z})
data_pivoted = plot_data.pivot("Month", "Year", "True/False")
ax = sns.heatmap(data_pivoted)
plt.show()
Prediction the crude oil price for next 6 months¶
Predicting prices for Cushing, OK WTI Spot Price FOB in Barrels
In [65]:
#Check for stationarity of the time series
from statsmodels.tsa.stattools import adfuller
X = df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)']
result = adfuller(X)
print('ADF Statistic: %f' % result[0])
print('p-value: %f' % result[1])
As p-value > 0.05, we can say that the series is non stationary Lets take log of the Series and check if that is make it stationary
In [66]:
X = np.log(X)
result = adfuller(X)
print('ADF Statistic: %f' % result[0])
print('p-value: %f' % result[1])
Again, our hull hypotheisis will be rejected in this case. The series stays to be non-stationary.
Let's check the Trend, seasonality and residual of the time series
As the series is non-stationary, we can apply S-ARIMA model for forecasting the prices of the of Crude Oil at Cushing
In [67]:
#Divide information into training and testing data
y = df_crudeOil[:'2010'].resample('MS').mean()
df_train=y[:'2010']
df_test=y['2010':]
#Decompose to see the trend and seasonality
decomposition = sm.tsa.seasonal_decompose(df_train['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)'], model='multiplicative')
fig = decomposition.plot()
plt.show()
Trend and observed graph show similar trend, it will be a multiplicative model.
Here, we try to get the best possible values p,q, r for our seasonal ARIMA model (that is which has the least AIC value)
In [68]:
p = d = q = range(0, 2)
pdq = list(itertools.product(p, d, q))
seasonal_pdq = [(x[0], x[1], x[2], 12) for x in list(itertools.product(p, d, q))]
min_val=9999999
comment=""
for param in pdq:
for param_seasonal in seasonal_pdq:
mod = sm.tsa.statespace.SARIMAX(df_train[['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)']],
order=param,
seasonal_order=param_seasonal,
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
#print('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic))
if(int(results.aic)<min_val):
min_val=int(results.aic)
comment=('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic))
print(str("Minimum AIC value is obtained when: ")+comment)
Using the min AIC value found combinations of p,q,r for training S-ARIMA model
In [69]:
mod = sm.tsa.statespace.SARIMAX(df_train[['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)']],
order=(1, 1, 1),
seasonal_order=(0, 1, 1, 12),
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
print(results.summary().tables[1])
In [70]:
results.plot_diagnostics(figsize=(16, 8))
plt.show()
In [71]:
pred = results.get_prediction(start=pd.to_datetime('2010-01-01'), dynamic=False)
pred_ci = pred.conf_int()
ax = y[['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)']].plot(label='Cushing, OK WTI Spot Price FOB (Dollars per Barrel)')
pred.predicted_mean.plot(ax=ax, label='One-step ahead Forecast', alpha=.7, figsize=(14, 7))
ax.fill_between(pred_ci.index,
pred_ci.iloc[:, 0],
pred_ci.iloc[:, 1], color='k', alpha=.2)
ax.set_xlabel('Date')
ax.set_ylabel('Dollars per Barrel')
plt.legend()
plt.show()
rms = sqrt(mean_squared_error(df_test[['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)']], pred.predicted_mean))
print("Root mean squered error: "+str(rms))
The visualization shows that the predictation was a correctede made. Therefore, we move ahead for training the model now with the complete dataset.
In [72]:
mod = sm.tsa.statespace.SARIMAX(df_crudeOil[['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)']],
order=(1, 1, 1),
seasonal_order=(0, 1, 1, 12),
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
print(results.summary().tables[1])
results.plot_diagnostics(figsize=(16, 8))
plt.show()
Again, everything seems good.
In [73]:
#Predict for next 6 months
pred = results.forecast(6)
pred=pd.DataFrame(pred)
date_list=['2019-02-15','2019-03-15','2019-04-15','2019-05-15','2019-06-15','2019-07-15']
pred.index=pd.to_datetime(date_list)
trace0 = go.Scatter(
x = df_crudeOil.index,
y = df_crudeOil['Cushing, OK WTI Spot Price FOB (Dollars per Barrel)'],
name = 'Cushing, OK WTI-Crude Oil',
line = dict(
color = ('rgb(166,206,227)'),
width = 2)
)
trace1 = go.Scatter(
x = pred.index,
y = pred[0],
name = 'Forecasted',
line = dict(
color = ('blue'),
width = 2)
)
data = [trace0,trace1]
layout = dict(title = 'Crude Oil - Cushing, OK WTI Spot Price FOB Forecast for next 6 months',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Barrel'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')
The forcasts seems to be aligned with it previous value
Predicting Price for Crude at Europe Bent in Barrels
In [74]:
#X=pd.DataFrame(df_crudeOil['Europe Brent Spot Price FOB (Dollars per Barrel)'])
X=[x for x in df_crudeOil['Europe Brent Spot Price FOB (Dollars per Barrel)'] if np.isnan(x)!=True]
result = adfuller(X)
print('ADF Statistic: %f' % result[0])
print('p-value: %f' % result[1])
Series is non - stationary, p-value > 0.05
In [75]:
X = np.log(X)
result = adfuller(X)
print('ADF Statistic: %f' % result[0])
print('p-value: %f' % result[1])
Log values of time series, also says that it is non-stationary
Predict the prices using seasonal ARIMA
Lets start with seeing the decomposition of data
In [76]:
y = df_crudeOil[:'2010'].resample('MS').mean()
y=y.dropna()
df_train=y[:'2010']
df_test=y['2010':]
df_train
#Decompose to see the trend and seasonality
decomposition = sm.tsa.seasonal_decompose(df_train['Europe Brent Spot Price FOB (Dollars per Barrel)'], model='multiplicative')
fig = decomposition.plot()
plt.show()
In [77]:
p = d = q = range(0, 2)
pdq = list(itertools.product(p, d, q))
seasonal_pdq = [(x[0], x[1], x[2], 12) for x in list(itertools.product(p, d, q))]
min_val=9999999
comment=""
for param in pdq:
for param_seasonal in seasonal_pdq:
mod = sm.tsa.statespace.SARIMAX(df_train[['Europe Brent Spot Price FOB (Dollars per Barrel)']],
order=param,
seasonal_order=param_seasonal,
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
#print('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic))
if(int(results.aic)<min_val):
min_val=int(results.aic)
comment=('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic))
print(str("Minimum AIC value is obtained when: ")+comment)
Similar to what we did before, we take the found combination and feed it to the SARIMA Model and its diagnostics
In [78]:
mod = sm.tsa.statespace.SARIMAX(df_train[['Europe Brent Spot Price FOB (Dollars per Barrel)']],
order=(1, 1, 1),
seasonal_order=(0, 1, 1, 12),
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
print(results.summary().tables[1])
results.plot_diagnostics(figsize=(16, 8))
plt.show()
Everything seems good, so we move ahead with the visualizing the predictions
In [79]:
pred = results.get_prediction(start=pd.to_datetime('2010-01-01'), dynamic=False)
pred_ci = pred.conf_int()
ax = y[['Europe Brent Spot Price FOB (Dollars per Barrel)']].plot(label='Europe Brent Spot Price FOB (Dollars per Barrel)')
pred.predicted_mean.plot(ax=ax, label='One-step ahead Forecast', alpha=.7, figsize=(14, 7))
ax.fill_between(pred_ci.index,
pred_ci.iloc[:, 0],
pred_ci.iloc[:, 1], color='k', alpha=.2)
ax.set_xlabel('Date')
ax.set_ylabel('Price in Barrels')
plt.legend()
plt.show()
rms = sqrt(mean_squared_error(df_test[['Europe Brent Spot Price FOB (Dollars per Barrel)']], pred.predicted_mean))
print("Root mean squered error: "+str(rms))
The model could correctly get the trend and predict the results. Now, we will train the model with complete data.
In [80]:
mod = sm.tsa.statespace.SARIMAX(df_crudeOil[['Europe Brent Spot Price FOB (Dollars per Barrel)']],
order=(1, 1, 1),
seasonal_order=(0, 1, 1, 12),
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
print(results.summary().tables[1])
results.plot_diagnostics(figsize=(16, 8))
plt.show()
All seems good, we therefore, visualize the forecasted data
In [81]:
#Predict for next 6 months
pred = results.forecast(6)
pred=pd.DataFrame(pred)
date_list=['2019-02-15','2019-03-15','2019-04-15','2019-05-15','2019-06-15','2019-07-15']
pred.index=pd.to_datetime(date_list)
trace0 = go.Scatter(
x = df_crudeOil.index,
y = df_crudeOil['Europe Brent Spot Price FOB (Dollars per Barrel)'],
name = 'Europe Brent - Crude Oil',
line = dict(
color = ('rgb(166,206,227)'),
width = 2)
)
trace1 = go.Scatter(
x = pred.index,
y = pred[0],
name = 'Forecasted',
line = dict(
color = ('blue'),
width = 2)
)
data = [trace0,trace1]
layout = dict(title = 'Crude Oil - Europe Brent Spot Price FOB Forecast for next 6 months',
xaxis = dict(title = 'Year'),
yaxis = dict(title = 'Dollars per Barrel'),
)
fig = dict(data=data, layout=layout)
offline.iplot(fig, filename='styled-line')